package com.ghl.crawler;

import com.ghl.crawler.core.DBHelper;
import com.ghl.crawler.entity.Item;
import com.ghl.crawler.mapper.ItemMapper;
import org.apache.ibatis.session.SqlSession;
import org.jsoup.Jsoup;
import org.jsoup.internal.StringUtil;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;

import java.io.IOException;
import java.util.ArrayList;
import java.util.Date;
import java.util.List;

/**
 * @Description
 * @Author haiLong <1105744725@qq.com>
 * @Version V1.0.0
 * @Since 1.0
 * @Date 2019/11/24 0:49
 */
public class App {

    private static final List<String> allUrl = new ArrayList();

    private static final List<String> level1 = new ArrayList();


    private static final String URL = "http://www.bytravel.cn/";

    private static final String[] USERAGENT = {
            "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/51.0.2704.106 Safari/537.36",
            "Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10_6_8; en-us) AppleWebKit/534.50 (KHTML, like Gecko) Version/5.1 Safari/534.50",
            "Mozilla/5.0 (Windows; U; Windows NT 6.1; en-us) AppleWebKit/534.50 (KHTML, like Gecko) Version/5.1 Safari/534.50",
            "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.6; rv:2.0.1) Gecko/20100101 Firefox/4.0.1", "Mozilla/5.0 (Windows NT 6.1; rv:2.0.1) Gecko/20100101 Firefox/4.0.1",
            "Mozilla/5.0 (iPhone; CPU iPhone OS 11_0 like Mac OS X) AppleWebKit/604.1.38 (KHTML, like Gecko) Version/11.0 Mobile/15A372 Safari/604.1"
    };


    public static void main(String[] args) {

        SqlSession session = DBHelper.getSession();
        ItemMapper itemMapper = session.getMapper(ItemMapper.class);
        //拿到主页面.
        Document document = getDocument(URL, USERAGENT);
        //获取所有地区的景点URL
        Elements elements = document.selectFirst("table").nextElementSibling().selectFirst("tr").selectFirst("td").selectFirst("table").nextElementSibling().select(".ht").select("a");

        for (Element element : elements) {
            String url = element.attr("abs:href");
            int index = url.lastIndexOf(".");
            StringBuilder sb = new StringBuilder(url);
            sb.insert(index, "_list");
            level1.add(sb.toString());
        }
        //遍历每个地区.获得对应的URL
        for (String url : level1) {
            try {
                //根据每个地区的URL.获取每个地区所有的URL
                System.out.println("-> 获取当前地区所有景点分页的url,当前url: " + url);
                List<String> urlPages = getPageListUrl(url);
                //遍历每个页面
                for (String urlPage : urlPages) {
                    Document doc = getDocument(urlPage, USERAGENT);
                    //获取详情页面所有展示出来的景点.每次做多20个
                    //http://www.bytravel.cn/view/index109_list.html
                    Elements details = doc.getElementById("page_left").getElementsByClass("blue14b");
                    String area = "";
                    Element child = doc.getElementById("page_left").selectFirst("div").child(0).child(0);
                    String area1 = child.child(1).text();
                    try {
                        area = area + area1;
                        String area2 = child.child(2).text();
                        area = area + area2;
                    } catch (Exception e) {
//                        e.printStackTrace();
                        area = area1;
                    }
                    for (Element element : details) {
                        // 暂停2秒后程序继续执行
                        Thread.sleep(1000);
                        String detailUrl = element.attr("abs:href");
                        System.out.println("-> 获取景点详情页内容: " + detailUrl);
                        //爬取每个详情页.
                        Document detailDoc = getDocument(detailUrl, USERAGENT);
                        //保存到数据库
                        insertItem(itemMapper, area, detailUrl, detailDoc);
                    }
                }
            } catch (Exception e) {
                e.printStackTrace();
            } finally {
                //清空集合
                allUrl.clear();
            }
        }
        session.close();

    }

    private static void insertItem(ItemMapper itemMapper, String area, String detailUrl, Document detailDoc) {
        try {
            //标题元素
            Element nameEle = detailDoc.getElementById("page_left").child(1);
            //内容元素
            Elements contentElements = detailDoc.getElementById("page_left").select("[style=margin:0 10px 0 10px;clear:both]");
            //图片元素
            Elements imgElements = detailDoc.getElementById("page_left").select("img");

            Item item = new Item();
            item.setSource(detailUrl.trim());
            item.setArea(area);
            item.setName(nameEle.text().trim());
            item.setContent(contentElements.text().trim());
            if (imgElements != null && imgElements.size() > 0) {
                item.setImage(imgElements.get(0).attr("abs:src").trim());
            }
            item.setCreateTime(new Date());
            itemMapper.insertItem(item);
        } catch (Exception e) {
            e.printStackTrace();
        }
    }


    /**
     * 获取文档对象
     *
     * @param url
     * @param agent
     * @return
     */
    public static Document getDocument(String url, String[] agent) {
        try {
            return Jsoup.connect(url).header("User-Agent", agent[(int) Math.random() * 5]).get();
        } catch (IOException e) {
            e.printStackTrace();
        }
        return null;
    }


    /**
     * 递归获取分页的所有的URL
     *
     * @param url
     * @return
     */
    public static List<String> getPageListUrl(String url) throws InterruptedException {
        allUrl.add(url);
        // 暂停2秒后程序继续执行
        Thread.sleep(1000);
        Document document = getDocument(url, USERAGENT);
        Elements elements = document.select("#list-page a:contains(下一页)");
        String currentUrl = elements.attr("abs:href");
        if (elements.size() == 0) {
            return allUrl;
        }
        if (StringUtil.isBlank(currentUrl)) {
            return allUrl;
        }
        getPageListUrl(currentUrl);
        return allUrl;
    }


}
